
# Carga de paquetes
from selenium                          import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by      import By
from selenium.webdriver.support.ui     import WebDriverWait
from selenium.webdriver.support        import expected_conditions as EC

from bs4                               import BeautifulSoup
import re
import pandas               as pd
import numpy                as np
import time
import plotly.express       as px
import plotly.graph_objects as go


options = Options()
options.headless = True
options.add_argument("--window-size=1920,1200")

DRIVER_PATH = './chromedriver.exe' # El driver para este ejemplo se encuentra en la misma carpeta de proyecto.


driver = webdriver.Chrome(executable_path=DRIVER_PATH)
driver.get('http://www.vigiaccess.org/')
print(driver.title)
time.sleep(5)

VigiAccess


# Recuerde salir de la consola de desarollo antes de su ejecución
driver.find_element_by_id('acceptTermsCheckBox').click()
driver.find_element_by_class_name('btn').click()
time.sleep(5)


# Recuerde salir de la consola de desarollo de Chrome antes de la ejecución de esta celda
drug = 'Ciprofloxacin'
driver.find_element_by_xpath("//input[@type='search']").send_keys(drug)
driver.find_element_by_xpath("//button[@ng-click='getDrug(drug)']").click()
time.sleep(5)


# Recuerde salir de la consola de desarollo de Chrome antes de la ejecución de esta celda
# Apertura de acordeón con Categorías de RAM
driver.find_element_by_xpath("//a[@class='accordion-toggle' and ./span[contains(text(),'Adverse drug reactions (ADRs)')]]").click()
RAM_categories = driver.page_source


# Apertura de nodos principales
category_ls = driver.find_elements_by_xpath("//span[@class='ng-binding ng-scope']")

for i in category_ls:
    i.click()
    time.sleep(1)


more_data_ls = [1]

while(len(more_data_ls) > 0):
    more_data_ls = driver.find_elements_by_xpath("//a[@class=\'ng-scope\']")
    # print(more_data_ls)
    for i in more_data_ls:
        try:
            i.click()
        except:
            print(i)

<selenium.webdriver.remote.webelement.WebElement (session="61521cdadcb82ced60f8635908ca33a2", element="25d8ff24-c616-4fcc-a09c-97a8cba3ac52")>
<selenium.webdriver.remote.webelement.WebElement (session="61521cdadcb82ced60f8635908ca33a2", element="4d9bcfa1-947f-4e4e-9dd7-58bafdc67ca2")>
<selenium.webdriver.remote.webelement.WebElement (session="61521cdadcb82ced60f8635908ca33a2", element="96bb39b4-07fd-41de-82b2-6e45912add29")>
<selenium.webdriver.remote.webelement.WebElement (session="61521cdadcb82ced60f8635908ca33a2", element="0318348b-8fb9-4c05-93e0-7d664b4e5baa")>
<selenium.webdriver.remote.webelement.WebElement (session="61521cdadcb82ced60f8635908ca33a2", element="a9eb196c-1336-493c-84ab-f258804f8d58")>
<selenium.webdriver.remote.webelement.WebElement (session="61521cdadcb82ced60f8635908ca33a2", element="6566b1ef-9e72-4544-a092-0621fd7a5cfc")>
<selenium.webdriver.remote.webelement.WebElement (session="61521cdadcb82ced60f8635908ca33a2", element="da217c27-99f3-4d68-b4b4-19e8b9ba3d08")>
<selenium.webdriver.remote.webelement.WebElement (session="61521cdadcb82ced60f8635908ca33a2", element="e1626ba0-3399-48d8-ae11-c7d9f5953800")>
<selenium.webdriver.remote.webelement.WebElement (session="61521cdadcb82ced60f8635908ca33a2", element="bdba1c6d-84c7-48f1-8e72-458d9638fde3")>
<selenium.webdriver.remote.webelement.WebElement (session="61521cdadcb82ced60f8635908ca33a2", element="4678ffaa-b571-424c-80a5-3b9b625be9b0")>
<selenium.webdriver.remote.webelement.WebElement (session="61521cdadcb82ced60f8635908ca33a2", element="baf5cb81-1c9a-4e72-b726-6df4498ae39a")>
<selenium.webdriver.remote.webelement.WebElement (session="61521cdadcb82ced60f8635908ca33a2", element="de23e6c7-7e5f-4069-9b9b-d3d890a267a3")>
<selenium.webdriver.remote.webelement.WebElement (session="61521cdadcb82ced60f8635908ca33a2", element="90e9d1f5-ee38-4261-84e3-a7dac032c5c9")>
<selenium.webdriver.remote.webelement.WebElement (session="61521cdadcb82ced60f8635908ca33a2", element="55d968c6-c5bd-4688-882b-f60bf98e09ff")>
<selenium.webdriver.remote.webelement.WebElement (session="61521cdadcb82ced60f8635908ca33a2", element="b3401322-063b-4cf8-b8a7-54d1f4232590")>
<selenium.webdriver.remote.webelement.WebElement (session="61521cdadcb82ced60f8635908ca33a2", element="992ee85b-5077-42f3-9513-70c0c6f9fbdc")>
<selenium.webdriver.remote.webelement.WebElement (session="61521cdadcb82ced60f8635908ca33a2", element="9bade87b-9b6b-4b1f-ba48-a98bb62d3696")>
<selenium.webdriver.remote.webelement.WebElement (session="61521cdadcb82ced60f8635908ca33a2", element="f3f1d1fe-ac22-446f-a9bb-18cfa0ee4522")>


page_source = driver.page_source

driver.quit()


soup = BeautifulSoup(page_source, 'lxml')


ls = [k.text for k in soup.find_all('span', {'class':'ng-binding'})]
ls[:5]

['Blood and lymphatic system disorders (3396)',
 'Thrombocytopenia (821)',
 'Leukopenia (516)',
 'Neutropenia (459)',
 'Eosinophilia (322)']


# Conversión de cada registro en una lista con la RAM y la frecuencia
ls_RAM = [[k.strip() for k in re.split('[\(\)]', m) if k!=''] for m in ls]
ls_RAM[:5]

[['Blood and lymphatic system disorders', '3396'],
 ['Thrombocytopenia', '821'],
 ['Leukopenia', '516'],
 ['Neutropenia', '459'],
 ['Eosinophilia', '322']]


# Conversión a DataFrame
RAM_DF = pd.DataFrame([{'RAM':k[0], 'Number':k[1]} for k in ls_RAM])


ram_cat_ls = [k.text for k in BeautifulSoup(RAM_categories, 'lxml').find_all('span', {'class':'ng-binding'})]
ram_cat_ls[:5]

['Blood and lymphatic system disorders (3396)',
 'Cardiac disorders (3390)',
 'Congenital, familial and genetic disorders (197)',
 'Ear and labyrinth disorders (2014)',
 'Endocrine disorders (194)']


ram_cat_ls1 = [[k.strip() for k in re.split('[\(\)]', m) if k!=''] for m in ram_cat_ls]
ram_cat_ls1 = [i[0] for i in ram_cat_ls1]
ram_cat_ls1[:5]

['Blood and lymphatic system disorders',
 'Cardiac disorders',
 'Congenital, familial and genetic disorders',
 'Ear and labyrinth disorders',
 'Endocrine disorders']


RAM_DF = RAM_DF.assign(
    Category = lambda df: df['RAM'].isin(ram_cat_ls1),
    RAM_name = lambda df: ~df['RAM'].isin(ram_cat_ls1)
)


RAM_DF['Category'] = list( map(lambda x,y: y if x else np.nan, RAM_DF['Category'], RAM_DF['RAM']) )
RAM_DF['RAM_name'] = list( map(lambda x,y: y if x else np.nan, RAM_DF['RAM_name'], RAM_DF['RAM']) )
RAM_DF['Category'] = RAM_DF['Category'].fillna(method='ffill')
RAM_DF = RAM_DF.dropna(0).drop(columns='RAM')
RAM_DF


RAM_DF.to_csv('./RAM_MORFINA.csv', index=False)


if True:
    RAM_DF = pd.read_csv('./RAM_MORFINA.csv')


RAM_DF


RAM_DF1 = RAM_DF\
.assign(RAM_name1 = lambda df: [i for i in map(lambda x,y: y if x > 15 else 'Others', RAM_DF.Number, RAM_DF.RAM_name)])\
.groupby(by = ['Category', 'RAM_name1']).sum().reset_index()

fig = px.treemap(RAM_DF1, path=['Category', 'RAM_name1'], values='Number',
                 color='Number', 
                 color_continuous_scale='viridis')

fig.data[0].hovertemplate = 'Id: %{label}<br>N: %{value:.0f}<br>Parent: %{parent}'

fig.show()


def colocarSitioPrimerNivel(driver, drug:str, report:str, url = 'http://www.vigiaccess.org/'):
    """ Este método toma el driver, abre la página "vigiaccess.org", realiza la búsqueda del 
    fármaco, y abre una sección.
    Args: 
        driver::selenium.webdriver.chrome.webdriver.WebDriver 
            Objeto webdriver de Selenium (para este tutorial Chrome)
        drug::str
            Fármaco a buscar en la base Vigiaccess
        report::str
            Tipo de reporte a consultar se pueden utilizar: (1) Adverse drug reactions (ADRs),
            (2) Geographical distribution, (3) Age group distribution, (4) Patient sex 
            distribution, y (5) ADR reports per year.
        url::str
            Sitio web de VigiAccess
    Returns:
        Nada, sólo modifica el objeto driver
    """
    # Abrir página
    driver.get(url)
    time.sleep(1)
    # Condiciones y términos
    driver.find_element_by_id('acceptTermsCheckBox').click()
    driver.find_element_by_class_name('btn').click()
    time.sleep(1)
    # Búsqueda de principio activo
    driver.find_element_by_xpath("//input[@type='search']").send_keys(drug)
    driver.find_element_by_xpath("//button[@ng-click='getDrug(drug)']").click()
    # Apertura de módulo 
    searchString1 = f"//a[@class='accordion-toggle' and ./span[contains(text(), '{report}')]]"
    time.sleep(5)
    el = driver.find_element_by_xpath(searchString1)
    el.click()


driver = webdriver.Chrome(executable_path=DRIVER_PATH)
colocarSitioPrimerNivel(driver, drug, 'ADR reports per year')
table = BeautifulSoup(driver.page_source, 'lxml').find('table', {'id':'yearTable'})
driver.quit()


# Conversión de la tabla HTML a un objeto DataFrame
table_rows = table.find_all('tr')

res = []
for tr in table_rows:
    td = tr.find_all('td')
    row = [tr.text.strip() for tr in td if tr.text.strip()]
    if row:
        res.append(row)
        
df = pd.DataFrame(res, columns=["Year", "Count", "Percentage"])
df.head()


fig = px.scatter(df, x="Year", y="Count", title=f"RAM por año para {drug}o", template='plotly_white')

fig.data[0].update(mode='markers+lines')
fig.update_layout(xaxis_title='Año', yaxis_title="Conteo RAM")
fig.update_traces(marker = {'size':10, 'line': {'width': 2, 'color': 'black'}})
fig.data[0].hovertemplate = 'Año: %{x}<br>Frec.: %{y:.0f}'
fig.show()


driver = webdriver.Chrome(executable_path=DRIVER_PATH)
colocarSitioPrimerNivel(driver, drug, 'Patient sex distribution')
table = BeautifulSoup(driver.page_source, 'lxml').find('table', {'id':'sexTable'})
driver.quit()


table_rows = table.find_all('tr')

res = []
for tr in table_rows:
    td = tr.find_all('td')
    row = [tr.text.strip() for tr in td if tr.text.strip()]
    if row:
        res.append(row)
        
df = pd.DataFrame(res, columns=['Sex', 'Count', 'Percentage'])
df.head()


# Traducción a español
sexo_dict = {'Female': 'Mujer', 'Male': 'Hombre', 'Unknown': 'Desconocido'}
df1 = df.assign(Sexo = lambda df: df.Sex.apply(lambda x: sexo_dict[x]))

fig = px.pie(df1, 
             values='Count', names='Sexo', title=f'RAM por sexo para {drug}o', 
             color_discrete_sequence=px.colors.sequential.RdBu)

fig.data[0].hovertemplate = 'Sexo: %{label}<br>Frecuencia: %{value:.0f}'
fig.show()


driver = webdriver.Chrome(executable_path=DRIVER_PATH)
colocarSitioPrimerNivel(driver, drug, 'Age group distribution')
table = BeautifulSoup(driver.page_source, 'lxml').find('table', {'id':'ageTable'})
driver.quit()


table_rows = table.find_all('tr')

res = []
for tr in table_rows:
    td = tr.find_all('td')
    row = [tr.text.strip() for tr in td if tr.text.strip()]
    if row:
        res.append(row)
        
df = pd.DataFrame(res, columns=['Age Group', 'Count', 'Percentage'])
df.head()


edad_dict = {
    '0 - 27 days':          ['0 - 27 días', 'Neonato'],
    '28 days to 23 months': ['28 días a 23 meses', 'Lactante'],
    '2 - 11 years':         ['2 a 11 años', 'Infante'],
    '12 - 17 years':        ['12 a 17 años', 'Adolescentes'],
    '18 - 44 years':        ['18 a 44 años', 'Adultos jóvenes'],
    '45 - 64 years':        ['45 a 64 años', 'Adultos'],
    '65 - 74 years':        ['65 a 74 años', 'Edad media'],
    '≥ 75 years':           ['≥ 75 años', 'Ancianos'],
    'Unknown':              ['Desconocido', 'Desconocido']
}

df1 = df.assign(
    Grupo_Edad = lambda df: df.loc[:, 'Age Group'].apply(lambda x: edad_dict[x][0]),
    Denom_Edad = lambda df: df.loc[:, 'Age Group'].apply(lambda x: edad_dict[x][1])
)

fig = px.bar(df1, 
            x='Count', y='Grupo_Edad', title=f'RAM por grupo etáreo para {drug}o', 
            color = 'Denom_Edad',
            color_discrete_sequence=px.colors.sequential.RdBu, 
            orientation='h', 
            template='plotly_white',
            text = 'Denom_Edad')

fig.update_layout(xaxis_title='Frecuencia', 
                  yaxis_title="Grupo de edad", 
                  legend_title_text='Grupo etáreo')

fig.update_traces(marker = {'line_color': 'black', 'line_width': 1.5}, 
                  hovertemplate = 'Grupo edad: %{y}<br>Conteo: %{x:.0f}')

fig.show()


driver = webdriver.Chrome(executable_path=DRIVER_PATH)
colocarSitioPrimerNivel(driver, drug, 'Geographical distribution')
table = BeautifulSoup(driver.page_source, 'lxml').find('table', {'id':'continentTable'})
driver.quit()


table_rows = table.find_all('tr')

res = []
for tr in table_rows:
    td = tr.find_all('td')
    row = [tr.text.strip() for tr in td if tr.text.strip()]
    if row:
        res.append(row)
        
df = pd.DataFrame(res, columns=['Continent', 'Count', 'Percentage'])
df.head()


gapminder = px.data.gapminder()\
    .loc[:, ['country', 'continent', 'iso_alpha']].drop_duplicates()\
    .merge(df, how='left', left_on='continent', right_on='Continent')
# gapminder.join(how='left')
gapminder


fig = px.choropleth(gapminder, locations='iso_alpha', color='Count', projection='natural earth', 
                    color_continuous_scale=px.colors.sequential.Plasma, 
                    title=f'RAM por origen de reporte para {drug}o', 
                    custom_data = ['continent', 'country', 'Count'])

fig.update_layout(legend_title_text='Conteo')

fig.update_traces(
    hovertemplate = 'Continente: %{customdata[0]}<br>País: %{customdata[1]}<br>Conteo: %{customdata[2]}')

fig.show()

	Number	Category	RAM_name
1	821	Blood and lymphatic system disorders	Thrombocytopenia
2	516	Blood and lymphatic system disorders	Leukopenia
3	459	Blood and lymphatic system disorders	Neutropenia
4	322	Blood and lymphatic system disorders	Eosinophilia
5	302	Blood and lymphatic system disorders	Anaemia
...	...	...	...
4977	1	Vascular disorders	Superior vena cava occlusion
4978	1	Vascular disorders	Varicose vein ruptured
4979	1	Vascular disorders	Vascular fragility
4980	1	Vascular disorders	Vein rupture
4981	1	Vascular disorders	Venoocclusive disease

	Number	Category	RAM_name
0	821	Blood and lymphatic system disorders	Thrombocytopenia
1	516	Blood and lymphatic system disorders	Leukopenia
2	459	Blood and lymphatic system disorders	Neutropenia
3	322	Blood and lymphatic system disorders	Eosinophilia
4	302	Blood and lymphatic system disorders	Anaemia
...	...	...	...
4950	1	Vascular disorders	Superior vena cava occlusion
4951	1	Vascular disorders	Varicose vein ruptured
4952	1	Vascular disorders	Vascular fragility
4953	1	Vascular disorders	Vein rupture
4954	1	Vascular disorders	Venoocclusive disease

	Year	Count	Percentage
0	2021	1007	1
1	2020	9737	8
2	2019	12151	11
3	2018	9900	9
4	2017	10141	9

	Continent	Count	Percentage
0	Africa	2000	2
1	Americas	34327	30
2	Asia	44548	39
3	Europe	32078	28
4	Oceania	1996	2

	country	continent	iso_alpha	Continent	Count	Percentage
0	Afghanistan	Asia	AFG	Asia	44548	39
1	Albania	Europe	ALB	Europe	32078	28
2	Algeria	Africa	DZA	Africa	2000	2
3	Angola	Africa	AGO	Africa	2000	2
4	Argentina	Americas	ARG	Americas	34327	30
...	...	...	...	...	...	...
137	Vietnam	Asia	VNM	Asia	44548	39
138	West Bank and Gaza	Asia	PSE	Asia	44548	39
139	Yemen, Rep.	Asia	YEM	Asia	44548	39
140	Zambia	Africa	ZMB	Africa	2000	2
141	Zimbabwe	Africa	ZWE	Africa	2000	2

Tutorial de Web Scrapping con Seleniun¶

^{Extracción de perfil de RAM de Sitio Web de VigiAccess}

1. Introducción¶

¿Qué es el Web Scrapping ?¶

¿Qué es VigiAccess?¶

Instalación de Selenium¶

2. Preparación de página con Selenium¶

2.1. Inicio VigiAccess Aceptación de Términos y Condiciones¶

2.2. Búsqueda de fármaco¶

1.3. Apertura de Nodos¶

2. Conversión de información de RAM a formato tabular¶

2.1. Revisión Categorías RAM¶

3. Extracción de reacciones por año¶

4. Extracción de RAM por sexo¶

5. Extracción de distribución de edad¶

6. Extracción de Continente de Origen RAM¶

Conclusión¶

	Age Group	Count	Percentage
0	0 - 27 days	116	0
1	28 days to 23 months	352	0
2	2 - 11 years	1214	1
3	12 - 17 years	1874	2
4	18 - 44 years	32535	28

	Sex	Count	Percentage
0	Female	65096	57
1	Male	45742	40
2	Unknown	4111	4